
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import datetime
import pickle
import pandas as pd
from pandas.io.parsers import read_csv
import random
import imgaug.augmenters as iaa
import imgaug as ia
import cv2 as cv
print("cv version (tested on 4.5.1) is {}".format(cv.__version__))
import tensorflow as tf
print("tf version (tested on 2.4.1) is {}".format(tf.__version__))

from sklearn.utils import shuffle
import datetime

from tensorflow.keras import Model
from tensorflow.keras.models import Sequential, save_model, load_model
from tensorflow.keras.losses import categorical_crossentropy
from tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPooling2D,Dropout
from tensorflow.keras.utils import to_categorical
import glob, os
from PIL import Image

notebook_root_path = os.getcwd()

print("current working directory : ", notebook_root_path)

cv version (tested on 4.5.1) is 4.5.1
tf version (tested on 2.4.1) is 2.4.1
current working directory :  /Users/giorgio/Documents/Udacity_SDC/P3-LeNet_traffic_signs/P3-Traffic-Sign-Classifier


plot_writeup_images = True
generate_fake_data_for_augmentation = True
train_model_flag = True
process_own_images = True


training_file = '../traffic-signs-data/train.p'
validation_file= '../traffic-signs-data/valid.p'
testing_file = '../traffic-signs-data/test.p'

with open(training_file, mode='rb') as f:
    train = pickle.load(f)
with open(validation_file, mode='rb') as f:
    valid = pickle.load(f)
with open(testing_file, mode='rb') as f:
    test = pickle.load(f)

X_train, y_train = train['features'], train['labels']
X_valid, y_valid = valid['features'], valid['labels']
X_test, y_test = test['features'], test['labels']

assert(len(X_train) == len(y_train))
assert(len(X_valid) == len(y_valid))
assert(len(X_test) == len(y_test))


def print_stats(x,y,text):
    """
    Helper function print simple dataset stats
    :param x: dataset features
    :param y: dataset labels
    :param text: simple text to print
    :return: none 
    """      
    print('Stats for the {} dataset'.format(text))
    assert(len(x) == len(y))
    # Number of training examples
    size = len(x)

    # What's the shape of a traffic sign image?
    index = random.randint(0, len(x))
    image = x[index]
    image_shape = image.shape

    # How many unique classes/labels there are in the dataset.
    if(len(y.shape)>1):
        n_classes = y.shape[1]
    else:
        n_classes = len(set(y))

    print('---------------------')
    print("Number of examples =", size)
    print("Image data shape =", image_shape)
    print("Number of classes =", n_classes)
    print('---------------------')

print_stats(X_train,y_train,'Training')    
print_stats(X_valid,y_valid,'Validation')
print_stats(X_test,y_test,'Testing')

Stats for the Training dataset
---------------------
Number of examples = 34799
Image data shape = (32, 32, 3)
Number of classes = 43
---------------------
Stats for the Validation dataset
---------------------
Number of examples = 4410
Image data shape = (32, 32, 3)
Number of classes = 43
---------------------
Stats for the Testing dataset
---------------------
Number of examples = 12630
Image data shape = (32, 32, 3)
Number of classes = 43
---------------------


def plot_sample_of_each(rows=3,columns=5):
    """
    Plots a subplot with rows*columns samples of images from the training dataset
    :return: saves a file
    """
    sign_names = pd.read_csv('signnames.csv')
    random_unique_sign_img = []
    sign_name_title = []
    df = pd.DataFrame({'sign':list(y_train)})
    classes = list(df.sign.unique())
    for sign in classes:
        
        unique_sign = df[df['sign']==sign].sample(n=1)
        random_unique_sign_img.append(X_train[unique_sign.index[0]])
        title = sign_names[sign_names['ClassId']==sign]['SignName'].values[0]
        if len(title.split())<5:
            sign_name_title.append(title)
        else:
            two_l = title.split()[0:4]
            two_l.extend('\n')
            two_l.extend(title.split()[4:])
            sign_name_title.append(' '.join(two_l))

    f, (axes) = plt.subplots(rows,columns, figsize=(32, 32)) 
    f.tight_layout()
    for i in range(rows):
        for j in range(columns):
            if(len(random_unique_sign_img)>0):
                axes[i][j].imshow(random_unique_sign_img.pop(0))
                axes[i][j].set_title(sign_name_title.pop(0),fontsize=30)
    plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.05,hspace = 0.4,wspace = 0.1)
    f.savefig('writeup_images/Some_example_sings_{}x{}.png'.format(rows,columns),dpi=300, bbox_inches = "tight")
    
    
if(plot_writeup_images):
    plot_sample_of_each(rows=3,columns=5)


def build_sign_dict(signnames,sign_classes,class_counts,labels):
    """
    Creates and plots an horizontal bar plot
    :param signnames: sorted list traffic signs 
    (index corresponds to the label)
    :param sign_classes: indices of the first element in the label list. 
    used as filter mask
    :param labels: dataset labels as (1D array)
    :return sign_dict: dict with sign names and values
    """    
    #get the size of the list 
    size = len(signnames)
    #prepare array
    filtered_counter = np.zeros(size)
    #get unique values
    unique_lab = set(labels)
    cnt = 0
    for i in range(size):
        if i in unique_lab:
            filtered_counter[cnt] = class_counts[cnt]
            cnt += 1
    
    #filter the sign names with existing labels   
    filtered_signnames  = signnames[sign_classes]
    
    #put sign names and counters into a dict
    sign_dict = {}
    for s,c in zip(filtered_signnames,filtered_counter):
        sign_dict[s]=c
        
    return sign_dict

def plot_sign_count_bar(signnames,labels,dataset_name='Training'):
    """
    Creates and plots an horizontal bar plot
    :param signnames: sorted list traffic signs 
    (index corresponds to the label)
    :param labels: list of labels, it can be in categorical form or 1D
    :param dataset_name: string used as filename title
    :return: saves a png file
    """
        
    # if label is converted to categorical
    # reverse it to 1D array
    if (len(labels.shape)>1):
        labels = np.argmax(labels,1)    
      
    #convert labels to int
    labels = [np.int(i) for i in labels]
    
    #sort labels
    labels = np.sort(labels)
    
    #get unique values and their position
    sign_classes, class_indices, class_counts = np.unique(
        labels, 
        return_index = True, 
        return_counts = True)    
    
    sign_dict = build_sign_dict(signnames,sign_classes,class_counts,labels)
    
    #sort dict values for nicer visualization
    values = sorted(sign_dict.values())
    labels = sorted(sign_dict, key=sign_dict.get)

    #generate figure
    plt.rcdefaults()
    fig, ax = plt.subplots()
    plt.tight_layout()
    fig.set_size_inches(15, 10.5)
    plt.margins(y=0.01)
    plt.grid(True)
    ax.barh(range(len(sign_dict)), values, align='center')
    bars = plt.barh(range(len(sign_dict)), values, align='center',color=['black','red'])
    plt.yticks(range(len(sign_dict)), labels)
    ax.invert_yaxis()  # labels read top-to-bottom
    ax.set_xlabel('Count')
    fig.savefig('writeup_images/{}_data_distrib.png'.format(dataset_name),
                dpi=300, 
                bbox_inches = "tight")
    
    
signnames = read_csv("signnames.csv").values[:, 1]

if(plot_writeup_images):
    #visualizing and saving Training data distribution
    plot_sign_count_bar(signnames,y_train,dataset_name='Training')    
    #visualizing and saving Validation data distribution
    plot_sign_count_bar(signnames,y_valid,dataset_name='Validation')   
    #visualizing and saving Test data distribution
    plot_sign_count_bar(signnames,y_test,dataset_name='Test')


def plot_samples_from_all_classes(images,labels,n_samples=10,color='rgb', show_indices=False):
    """
    Plots some random examples per each class of images
    :param images: list images
    :param labels: list of numerical labels
    :param n_samples: number of samples per class
    :param color: default is rgb, valid alternative is gray
    :return: none
    """
    # if label is converted to categorical
    # reverse it to 1D array
    if (len(labels.shape)>1):
        labels = np.argmax(labels,1)     
    
    #convert labels to int
    labels = [np.int(i) for i in labels]
    
    sign_classes, class_indices, class_counts = np.unique(labels, 
                                                          return_index = True, 
                                                          return_counts = True)
    df = pd.DataFrame({'sign':list(labels)})
    col_width = max(len(name) for name in signnames)
    for c, c_index, c_count in zip(sign_classes, class_indices, class_counts):
        print("Class %i: %-*s  %s samples" % (c, col_width, signnames[c], str(c_count)))
        fig = plt.figure(figsize = (6, 1))
        fig.subplots_adjust(left = 0, 
                            right = 1, 
                            bottom = 0, 
                            top = 1, 
                            hspace = 0.05, 
                            wspace = 0.05)
        if (n_samples>c_count):
            n_samples_show = c_count
        else:
            n_samples_show = n_samples
        random_indices = df[df['sign']==c].sample(n=n_samples_show)
        random_indices = random_indices.index[:]
        for i in range(n_samples_show):
            axis = fig.add_subplot(1, n_samples_show, i + 1, xticks=[], yticks=[])
            if color == 'gray':
                axis.imshow(images[random_indices[i]],cmap='gray')
            else:
                axis.imshow(images[random_indices[i]])
        plt.show()
        if(show_indices):
            print("Image indices: {}".format(random_indices))
        print("--------------------------------------------------------------------------------------\n")

    
if(plot_writeup_images):
    plot_samples_from_all_classes(X_train,y_train,show_indices=True)

Class 0: Speed limit (20km/h)                                180 samples

Image indices: Int64Index([10030, 10018, 10007, 9974, 10036, 9961, 10125, 10069, 10073,
            10090],
           dtype='int64')
--------------------------------------------------------------------------------------

Class 1: Speed limit (30km/h)                                1980 samples

Image indices: Int64Index([3937, 3643, 2588, 2452, 3697, 2854, 2414, 2618, 3770, 3521], dtype='int64')
--------------------------------------------------------------------------------------

Class 2: Speed limit (50km/h)                                2010 samples

Image indices: Int64Index([32186, 32137, 32073, 33302, 32345, 33275, 32829, 32011, 32648,
            33085],
           dtype='int64')
--------------------------------------------------------------------------------------

Class 3: Speed limit (60km/h)                                1260 samples

Image indices: Int64Index([5500, 5976, 5908, 5503, 6307, 5972, 6342, 6487, 5609, 6220], dtype='int64')
--------------------------------------------------------------------------------------

Class 4: Speed limit (70km/h)                                1770 samples

Image indices: Int64Index([7526, 6915, 7663, 8441, 7515, 7400, 8552, 7927, 8264, 7375], dtype='int64')
--------------------------------------------------------------------------------------

Class 5: Speed limit (80km/h)                                1650 samples

Image indices: Int64Index([12652, 12465, 13923, 13978, 12585, 13608, 12536, 12955, 12468,
            12835],
           dtype='int64')
--------------------------------------------------------------------------------------

Class 6: End of speed limit (80km/h)                         360 samples

Image indices: Int64Index([21534, 21647, 21727, 21597, 21769, 21536, 21671, 21496, 21590,
            21730],
           dtype='int64')
--------------------------------------------------------------------------------------

Class 7: Speed limit (100km/h)                               1290 samples

Image indices: Int64Index([24654, 23731, 24003, 24239, 24386, 24697, 24483, 24792, 24498,
            23867],
           dtype='int64')
--------------------------------------------------------------------------------------

Class 8: Speed limit (120km/h)                               1260 samples

Image indices: Int64Index([16136, 15927, 16832, 16465, 16901, 16273, 16137, 16420, 16103,
            16691],
           dtype='int64')
--------------------------------------------------------------------------------------

Class 9: No passing                                          1320 samples


def normalise_input(x,shift_for_plotting=False): 
    """
    Nromalize and center the distribution of image pixels 
    :param x: pixels
    :param shift_for_plotting: to be able to plot the effect in a plot, 
    the distribution should be positive, hence shift from [-1,1] to [0,1]
    :return: same image as input with normalised and centered pixel distrib.
    """
    # convert from integers to floats
    x = x.astype('float32')
    # calculate global mean and standard deviation
    mean, std = x.mean(), x.std()
    print('Mean: %.3f, Standard Deviation: %.3f' % (mean, std))
    # global standardization of pixels
    x = (x - mean) / std
    # clip pixel values to [-1,1]
    x = np.clip(x, -1.0, 1.0)
    if(shift_for_plotting):
        # shift from [-1,1] to [0,1] with 0.5 mean
        output = (x + 1.0) / 2.0
    else:
        output = x
    # confirm it had the desired effect
    mean, std = output.mean(), output.std()        
    print('New Mean: %.3f, Standard Deviation: %.3f' % (mean, std))    
    assert(x.shape==output.shape)
    return output


def image_crop(img,percent,pad_cval):
    """
    Applies the scale filter to the input image and it returns a copy 
    :param image: input image
    :param percent: The number of pixels to crop (negative values) or pad (positive values)
    on each side of the image given as a *fraction* of the image
    height/width.
    :param pad_cval: a tuple ``(a, b)``, then a value will be uniformly sampled
                          per image from the interval ``[a, b]`` 
    :return: copy of the image after processing
    """
    
    aug = iaa.CropAndPad(percent=percent,pad_cval=pad_cval)
    return aug(image=img)

def image_scale(img,scale_dict):
    """
    Applies the scale filter to the input image and it returns a copy 
    :param image: input image
    :param scale_dict: 
        {"x": number/tuple/list/StochasticParameter, 
        "y": number/tuple/list/StochasticParameter}
    :return: copy of the image after processing
    """
    
    aug = iaa.Affine(scale=scale_dict)
    return aug(image=img)

def image_translate(img,translate_dict):
    """
    Applies the translation filter to the input image and it returns a copy 
    :param image: input image
    :param translate_dict: 
        {"x": number/tuple/list/StochasticParameter, 
        "y": number/tuple/list/StochasticParameter}
    :return: copy of the image after processing
    """
    aug = iaa.Affine(translate_percent=translate_dict)
    return aug(image=img)


def image_rotate(img,rotate_tuple):
    """
    Applies the translation filter to the input image and it returns a copy 
    :param image: input image
    :param rotate_tuple: a tuple ``(a, b)``, then a value will be uniformly sampled
                          per image from the interval ``[a, b]`` and used as the rotation
                        value.
    :return: copy of the image after processing
    """
    aug = iaa.Affine(rotate=rotate_tuple)
    return aug(image=img)

def image_shear(img,shear_tuple):
    """
    Applies the translation filter to the input image and it returns a copy 
    :param image: input image
    :param shear_tuple: a tuple ``(a, b)``, then a value will be uniformly sampled
                          per image from the interval ``[a, b]`` and used as the shear
                        value.
    :return: copy of the image after processing
    """
    aug = iaa.Affine(shear=shear_tuple)
    return aug(image=img)



def applyCLAHE(img ):
    """
    Applies the CLAHE histogram equalization
    ref: https://docs.opencv.org/master/d5/daf/tutorial_py_histogram_equalization.html
    :param image: input image
    :return: copy of the image after processing
    """
    clahe = cv.createCLAHE()
    return clahe.apply(img)


def plot_writeup_preprocess_images():
    # choose one image 
    index = 553
    image = X_train[index]
    #grayscale
    grayimg = cv.cvtColor(image, cv.COLOR_BGR2GRAY)
    #apply clahe
    clahe = applyCLAHE(grayimg)
    #scale
    scaled = image_scale(grayimg,{"x": (0.5), "y": (0.5)})
    #tranlate
    translated = image_translate(grayimg,{"x": (0.2), "y": (0.2)})
    #rotate
    rotated = image_rotate(grayimg,(45))
    #shear
    sheared = image_shear(grayimg,(20))
    #crop
    cropped = image_crop(grayimg,(0.1),(0, 255))
    #normalise
    normalised = normalise_input(grayimg,shift_for_plotting=True)

    plt.figure(figsize=(1,1))
    print(signnames[y_train[index]])
    f,((ax1,ax2,ax3),(ax4,ax5,ax6),(ax7,ax8,ax9)) = plt.subplots(3,3,)
    f.tight_layout()

    ax1.imshow(image)
    ax1.set_title("Unprocessed image")

    ax2.imshow(grayimg,cmap='gray')
    ax2.set_title("Gray image")

    ax3.imshow(scaled,cmap='gray')
    ax3.set_title("Scaled image")

    ax4.imshow(translated,cmap='gray')
    ax4.set_title("Translated image")

    ax5.imshow(rotated,cmap='gray')
    ax5.set_title("Rotated image")

    ax6.imshow(sheared,cmap='gray')
    ax6.set_title("Sheared image")

    ax7.imshow(cropped,cmap='gray')
    ax7.set_title("Cropped image")

    ax8.imshow(clahe,cmap='gray')
    ax8.set_title("CLAHE filter")

    ax9.imshow(normalised,cmap='gray')
    ax9.set_title("Normalised image")

    plt.show()

    f.savefig('writeup_images/Preprocessing_img_{}.png'.format(index),dpi=300, bbox_inches = "tight")

if(plot_writeup_images):
    plot_writeup_preprocess_images()

Mean: 119.006, Standard Deviation: 29.175
New Mean: 0.565, Standard Deviation: 0.386
Wild animals crossing

<Figure size 100x100 with 0 Axes>


def writeup_clahe_images(): 
    # choose one image that is particularly dark 
    index = 12358
    image = X_train[index]
    grayimg = cv.cvtColor(image, cv.COLOR_BGR2GRAY)
    clahe = applyCLAHE(grayimg)

    plt.figure(figsize=(1,1))
    print(signnames[y_train[index]])
    f,((ax1,ax2,ax3)) = plt.subplots(1,3,)
    f.tight_layout()

    ax1.imshow(image)
    ax1.set_title("Unprocessed image")

    ax2.imshow(grayimg,cmap='gray')
    ax2.set_title("Gray image")

    ax3.imshow(clahe,cmap='gray')
    ax3.set_title("CLAHE image")

    f.savefig('writeup_images/histogram_equalization_image_{}.png'.format(index),dpi=300, bbox_inches = "tight")

if(plot_writeup_images):    
    writeup_clahe_images()

No passing

<Figure size 100x100 with 0 Axes>


def applyEqualizationAndGray(images,gray_image=False):
    """
    Applies the CLAHE histogram equalization
    ref: https://docs.opencv.org/master/d5/daf/tutorial_py_histogram_equalization.html
    :param image: list of images
    :param gray_image: flag, if true the input images are already in grayscale
    :return: copy of the image after processing
    """
    output = np.empty_like(images,shape=(len(images),32,32,1))
    
    for i in range(len(images)):    
        #gray image and CLAHE
        if(gray_image == False):
            grayimg = cv.cvtColor(images[i], cv.COLOR_BGR2GRAY)
        else:
            grayimg = images[i]
            
        output[i] = np.expand_dims(applyCLAHE(grayimg), axis=2) 
        
    return output


X_train_CLAHE_gray = applyEqualizationAndGray(X_train)

X_valid_CLAHE_gray = applyEqualizationAndGray(X_valid)

X_test_CLAHE_gray = applyEqualizationAndGray(X_test)
    
if(plot_writeup_images):
    plot_samples_from_all_classes(X_train_CLAHE_gray,y_train,color='gray')
    plot_samples_from_all_classes(X_valid_CLAHE_gray,y_valid,color='gray')
    plot_samples_from_all_classes(X_test_CLAHE_gray,y_test,color='gray')

Class 0: Speed limit (20km/h)                                180 samples

--------------------------------------------------------------------------------------

Class 1: Speed limit (30km/h)                                1980 samples

--------------------------------------------------------------------------------------

Class 2: Speed limit (50km/h)                                2010 samples

--------------------------------------------------------------------------------------

Class 3: Speed limit (60km/h)                                1260 samples

--------------------------------------------------------------------------------------

Class 4: Speed limit (70km/h)                                1770 samples

--------------------------------------------------------------------------------------

Class 5: Speed limit (80km/h)                                1650 samples

--------------------------------------------------------------------------------------

Class 6: End of speed limit (80km/h)                         360 samples

--------------------------------------------------------------------------------------

Class 7: Speed limit (100km/h)                               1290 samples

--------------------------------------------------------------------------------------

Class 8: Speed limit (120km/h)                               1260 samples

--------------------------------------------------------------------------------------

Class 9: No passing                                          1320 samples


def save_pickle_file(filename,features,labels):
    """
    Helper function to save features and lables as 
    a dict inside a pickle file
    :param filename: name of the pickle file
    :param features: features to save
    :param labels: labels to save
    :return: nothing
    """
    assert(len(features) == len(labels))
    data = {'features':features,'labels':labels}
    
    with open(filename, mode='wb') as f:
        pickle.dump(data, f)


# Sometimes(0.5, ...) applies the given augmenter in 50% of 
# all cases, e.g. Sometimes(0.5, GaussianBlur(0.3)) 
# would blur roughly every second image.
sometimes = lambda aug: iaa.Sometimes(0.5, aug)

#the sequence of filters to apply to the images
seq = iaa.Sequential(
    [
        iaa.SomeOf((1, 2),
            [
                # crop images by -10% to 10% of their height/width
                sometimes(iaa.CropAndPad(
                    percent=(-0.1, 0.1),
                    pad_cval=(0, 255)
                )),
                sometimes(iaa.Affine(
                    # scale between 80% and 120% of the image (per axis)
                    scale={"x": (0.8, 1.2), "y": (0.8, 1.2)},
                    # translate by -20 to +20 percent (per axis)
                    translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)}, 
                    # rotate by -45 to +45 degrees
                    rotate=(-45, 45), 
                    # shear by -20 to +20 degrees
                    shear=(-20, 20),  
                )),
            ],
            random_order=True
        )
    ],
    random_order=True
)
def populate_class_size_normalisation_table(item_counts):
    """
    Helper function that returns a dataframe with the info
    about which image class needs to augmented and how many 
    fake images need to be generated 
    :param item_counts: number of images per each class
    :return: dataframe 
    """  
    #save the size of the largest class
    max_val = np.max(item_counts)
    #create the dataframe
    aug_table = pd.DataFrame({'class_id':[i for i in range(43)],
                             'img_cnt':item_counts,
                             'should_be':max_val,
                             'to_add':0,
                             'class_aug_factor':0})
    
    #populate the dataframe by scanning the
    #input vector 
    for current_class, val in enumerate(item_counts):
        add = max_val - val

        aug_factor = int(np.floor(add/val))+1

        aug_table.at[current_class,'to_add']=add
        aug_table.at[current_class,'class_aug_factor']=aug_factor
   
    return aug_table

def augment(aug_table,images,labels,single_channel_img=True,augmentations=None,augment_percent=50):
    """
    Function that generates fake images. If augment_percent==0
    the function generates data so that all classes have the same
    number of images as the largest class of the dataset
    :param aug_table: dataframe that indicates how many 
    fake images per class must be generated
    :param images: original images
    :param labels: corresponding labels
    :param augmentations: imaug filters to apply for fake 
    image generation
    :param augment_percent: 
        0 means generate image to balance all classes
        100 means balance all classes and double each class size
    :return: new_images,new_lables
    """  
    new_images = np.array([])
    new_lables = np.array([])
    
    for row in range(aug_table.shape[0]):
        #current class
        current_class = aug_table.loc[row].at['class_id']
    
        # get the indexes of the images whose label
        # corresponds to current_class
        itemindex = np.where(labels==current_class)
        
        # select all images that correspond to the current class
        images_of_this_class = images[itemindex[0:2]]
        
        #current class population?
        class_size = aug_table.loc[row].at['img_cnt']
        
        assert(class_size == len(images_of_this_class))
        
        #target size for this class
        target_size = aug_table.loc[row].at['should_be']
        
        if (augment_percent>0):
            target_size = target_size + np.floor(target_size*augment_percent/100)
        
        print("augmenting class {} from {} to {} unique images".format(current_class,class_size,target_size))
        
        #until the target size is not reached generate new
        #fake images. If an image already exists, discard it 
        #and generate a new onw
        while (len(images_of_this_class)<target_size):
             
            #pick one of the original images
            random_img_idx = np.random.randint(0,class_size)
            
            #pick the image
            original = images_of_this_class[random_img_idx]

            #pass it to some filters
            edited_img = augmentations(image=original.squeeze())
            if(single_channel_img):
                edited_img = np.expand_dims(edited_img, axis=2)
            
            #check if edited image already exists, if not, 
            #add it to the array
            for i in range(len(images_of_this_class)):
                if np.array_equal(edited_img, images_of_this_class[i]):
                    #print("image already exist")
                    break

            #add the image and the label            
            edited_img = np.expand_dims(edited_img, axis=0)
            images_of_this_class = np.append(images_of_this_class,edited_img,axis=0)
            

        labels_of_this_class = np.array([current_class]*len(images_of_this_class))
        assert(len(labels_of_this_class) == len(images_of_this_class)) 
        
        #add new class images to the dataset
        if (new_images.shape[0]==0):
            new_images = images_of_this_class
            new_lables = labels_of_this_class
        else:
            new_images= np.append(new_images,images_of_this_class,axis=0)
            new_lables= np.append(new_lables,labels_of_this_class,axis=0)
    
    return new_images,new_lables


aug_table = None 
df = pd.DataFrame({'sign':list(y_train)}) 
item_counts = df["sign"].value_counts().sort_index()
aug_table = populate_class_size_normalisation_table(item_counts)

#0=only generate data to balance the datasets so 
# that all classes have the same size
#100= generate data to balance the datasets AND 
# increases the all classes by 100% (double the size)
fake_data_augmentation_factor = 100

if(generate_fake_data_for_augmentation):
    x_augmented,y_augmented = augment(
        aug_table,
        X_train,
        y_train,
        single_channel_img=False,
        augmentations=seq,
        augment_percent=fake_data_augmentation_factor)

    if(plot_writeup_images):
        plot_samples_from_all_classes(x_augmented,y_augmented)

        #visualizing and saving the new training data distribution
        plot_sign_count_bar(signnames,y_augmented,dataset_name='Training_balanced_aug_factor{}'.format(fake_data_augmentation_factor))

augmenting class 0 from 180 to 4020.0 unique images
augmenting class 1 from 1980 to 4020.0 unique images
augmenting class 2 from 2010 to 4020.0 unique images
augmenting class 3 from 1260 to 4020.0 unique images
augmenting class 4 from 1770 to 4020.0 unique images
augmenting class 5 from 1650 to 4020.0 unique images
augmenting class 6 from 360 to 4020.0 unique images
augmenting class 7 from 1290 to 4020.0 unique images
augmenting class 8 from 1260 to 4020.0 unique images
augmenting class 9 from 1320 to 4020.0 unique images
augmenting class 10 from 1800 to 4020.0 unique images
augmenting class 11 from 1170 to 4020.0 unique images
augmenting class 12 from 1890 to 4020.0 unique images
augmenting class 13 from 1920 to 4020.0 unique images
augmenting class 14 from 690 to 4020.0 unique images
augmenting class 15 from 540 to 4020.0 unique images
augmenting class 16 from 360 to 4020.0 unique images
augmenting class 17 from 990 to 4020.0 unique images
augmenting class 18 from 1080 to 4020.0 unique images
augmenting class 19 from 180 to 4020.0 unique images
augmenting class 20 from 300 to 4020.0 unique images
augmenting class 21 from 270 to 4020.0 unique images
augmenting class 22 from 330 to 4020.0 unique images
augmenting class 23 from 450 to 4020.0 unique images
augmenting class 24 from 240 to 4020.0 unique images
augmenting class 25 from 1350 to 4020.0 unique images
augmenting class 26 from 540 to 4020.0 unique images
augmenting class 27 from 210 to 4020.0 unique images
augmenting class 28 from 480 to 4020.0 unique images
augmenting class 29 from 240 to 4020.0 unique images
augmenting class 30 from 390 to 4020.0 unique images
augmenting class 31 from 690 to 4020.0 unique images
augmenting class 32 from 210 to 4020.0 unique images
augmenting class 33 from 599 to 4020.0 unique images
augmenting class 34 from 360 to 4020.0 unique images
augmenting class 35 from 1080 to 4020.0 unique images
augmenting class 36 from 330 to 4020.0 unique images
augmenting class 37 from 180 to 4020.0 unique images
augmenting class 38 from 1860 to 4020.0 unique images
augmenting class 39 from 270 to 4020.0 unique images
augmenting class 40 from 300 to 4020.0 unique images
augmenting class 41 from 210 to 4020.0 unique images
augmenting class 42 from 210 to 4020.0 unique images
Class 0: Speed limit (20km/h)                                4020 samples

--------------------------------------------------------------------------------------

Class 1: Speed limit (30km/h)                                4020 samples

--------------------------------------------------------------------------------------

Class 2: Speed limit (50km/h)                                4020 samples

--------------------------------------------------------------------------------------

Class 3: Speed limit (60km/h)                                4020 samples

--------------------------------------------------------------------------------------

Class 4: Speed limit (70km/h)                                4020 samples

--------------------------------------------------------------------------------------

Class 5: Speed limit (80km/h)                                4020 samples

--------------------------------------------------------------------------------------

Class 6: End of speed limit (80km/h)                         4020 samples

--------------------------------------------------------------------------------------

Class 7: Speed limit (100km/h)                               4020 samples

--------------------------------------------------------------------------------------

Class 8: Speed limit (120km/h)                               4020 samples

--------------------------------------------------------------------------------------

Class 9: No passing                                          4020 samples


X_train = normalise_input(X_train_CLAHE_gray)
X_valid = normalise_input(X_valid_CLAHE_gray)
X_test = normalise_input(X_test_CLAHE_gray)

print("min:{},max:{}".format(np.min(X_train),np.max(X_train)))
print("min:{},max:{}".format(np.min(X_valid),np.max(X_valid)))
print("min:{},max:{}".format(np.min(X_test),np.max(X_test)))

if(generate_fake_data_for_augmentation):
    X_train_aug_CLAHE_gray = applyEqualizationAndGray(x_augmented)
    X_train_aug_norm = normalise_input(X_train_aug_CLAHE_gray)
    print("min:{},max:{}".format(np.min(X_train_aug_norm),np.max(X_train_aug_norm)))

Mean: 132.128, Standard Deviation: 62.166
New Mean: -0.031, Standard Deviation: 0.759
Mean: 132.219, Standard Deviation: 62.615
New Mean: -0.033, Standard Deviation: 0.760
Mean: 132.785, Standard Deviation: 63.019
New Mean: -0.034, Standard Deviation: 0.761
min:-1.0,max:1.0
min:-1.0,max:1.0
min:-1.0,max:1.0
Mean: 127.834, Standard Deviation: 63.112
New Mean: -0.037, Standard Deviation: 0.766
min:-1.0,max:1.0


# save new pickle files, training dataset, 
# augmented training dataset, validation dataset 
# and testing dataset

filename= '../traffic-signs-data/train_clahe_norm_no_aug.p'
save_pickle_file(filename,X_train,y_train)

if(generate_fake_data_for_augmentation):
    filename = '../traffic-signs-data/train_CLAHE_norm_balanced_aug_factor_{}.p'.format(fake_data_augmentation_factor)
    save_pickle_file(filename,X_train_aug_norm,y_augmented)

filename= '../traffic-signs-data/valid_clahe_norm_no_aug.p'
save_pickle_file(filename,X_valid,y_valid)

filename= '../traffic-signs-data/test_clahe_norm_no_aug.p'
save_pickle_file(filename,X_test,y_test)


training_file= '../traffic-signs-data/train_clahe_norm_no_aug.p'
training_file_aug_0= '../traffic-signs-data/train_CLAHE_norm_balanced_aug_factor_0.p'
training_file_aug_100= '../traffic-signs-data/train_CLAHE_norm_balanced_aug_factor_100.p'
validation_file= '../traffic-signs-data/valid_clahe_norm_no_aug.p'
testing_file= '../traffic-signs-data/test_clahe_norm_no_aug.p'

with open(training_file, mode='rb') as f:
    train = pickle.load(f)
with open(training_file_aug_0, mode='rb') as f:
    train_aug_0 = pickle.load(f)
with open(training_file_aug_100, mode='rb') as f:
    train_aug_100 = pickle.load(f)    
with open(validation_file, mode='rb') as f:
    valid = pickle.load(f)
with open(testing_file, mode='rb') as f:
    test = pickle.load(f)
    
    
X_train, y_train = train['features'], train['labels']
X_train_aug_0, y_train_aug_0 = train_aug_0['features'], train_aug_0['labels']
X_train_aug_100, y_train_aug_100 = train_aug_100['features'], train_aug_100['labels']
X_valid, y_valid = valid['features'], valid['labels']
X_test, y_test = test['features'], test['labels']


def get_n_classes(labels):
    return len(set(labels))
    
print_stats(X_train,y_train,'original training')
print_stats(X_train_aug_0,y_train_aug_0,'augmented training')
print_stats(X_train_aug_100,y_train_aug_100,'augmented training')
print_stats(X_valid,y_valid,'validation')
print_stats(X_test,y_test,'testing')

Stats for the original training dataset
---------------------
Number of examples = 34799
Image data shape = (32, 32, 1)
Number of classes = 43
---------------------
Stats for the augmented training dataset
---------------------
Number of examples = 86430
Image data shape = (32, 32, 1)
Number of classes = 43
---------------------
Stats for the augmented training dataset
---------------------
Number of examples = 172860
Image data shape = (32, 32, 1)
Number of classes = 43
---------------------
Stats for the validation dataset
---------------------
Number of examples = 4410
Image data shape = (32, 32, 1)
Number of classes = 43
---------------------
Stats for the testing dataset
---------------------
Number of examples = 12630
Image data shape = (32, 32, 1)
Number of classes = 43
---------------------


# Convert class vectors to binary class matrices.

n_classes = get_n_classes(y_train)

y_train = to_categorical(y_train, n_classes)
y_train_aug_0 = to_categorical(y_train_aug_0, n_classes)
y_train_aug_100 = to_categorical(y_train_aug_100, n_classes)
y_valid = to_categorical(y_valid, n_classes)
y_test = to_categorical(y_test, n_classes)


assert(len(X_train) == len(y_train))
assert(len(X_train_aug_0) == len(y_train_aug_0))
assert(len(X_train_aug_100) == len(y_train_aug_100))
assert(len(X_valid) == len(y_valid))
assert(len(X_test) == len(y_test))


#shuffle data
X_train, y_train = shuffle(X_train, y_train)
X_train_aug_0, y_train_aug_0 = shuffle(X_train_aug_0, y_train_aug_0)
X_train_aug_100, y_train_aug_100 = shuffle(X_train_aug_100, y_train_aug_100)


#uncomment the dataset and label to use for training
#training_dataset_features = X_train
#training_dataset_labels = y_train

#training_dataset_features = X_train_aug_0
#training_dataset_labels = y_train_aug_0

training_dataset_features = X_train_aug_100
training_dataset_labels = y_train_aug_100

input_shape = training_dataset_features[0].shape

#dropout parameter for layer 4
p4 = .3
#dropout parameter for layer 5
p5 = .2
#kernel size of all conv 2D layers (layer 1,2 and 3)
conv_kern = 3
#depth of the conv 2D layers (layer 1,2 and 3)
conv_depths = [32,64,128]

#num epochs
training_epochs=200
#use early stop condition
early_stop_active = True
#training batch size
training_batch_size = 512

print_stats(training_dataset_features,training_dataset_labels,'selected training')

Stats for the selected training dataset
---------------------
Number of examples = 172860
Image data shape = (32, 32, 1)
Number of classes = 43
---------------------


# Modified LeNet-5 model with three convolutional layers
model = Sequential(name='c1{}_c2{}_c3{}_k{}_p4{}_p5{}_es{}'.format(
    conv_depths[0],
    conv_depths[1],
    conv_depths[2],
    conv_kern,
    p4,
    p5,
    early_stop_active))

#Layer 1: Convolutional. Input = 32x32x1. Output = 30x30x32.
model.add(
    Conv2D(
        conv_depths[0], 
        kernel_size=(conv_kern, conv_kern), 
        strides=(1, 1), 
        activation='relu', 
        input_shape=input_shape, 
        padding="valid",
        name='layer1'))

#Pooling. Input = 30x30x32. Output = 15x15x32.
model.add(
    MaxPooling2D(
        pool_size=(2, 2), 
        padding='same',
        name='maxPool1'))

#Layer 2: Convolutional. Output = 13x13x64.
model.add(
    Conv2D(
        conv_depths[1], 
        kernel_size=(conv_kern, conv_kern), 
        strides=(1, 1), 
        activation='relu', 
        padding='valid',
        name='layer2'))

#Pooling. Input = 13x13x64. Output = 7x7x64.
model.add(
    MaxPooling2D(
        pool_size=(2, 2), 
        padding='same',
        name='maxPool2'))

#Layer 3: Convolutional. Output = 5x5x128.
model.add(
    Conv2D(
        conv_depths[2], 
        kernel_size=(conv_kern, conv_kern), 
        strides=(1, 1), 
        activation='relu', 
        padding='valid',
        name='layer3'))

#Pooling. Input = 5x5x128. Output = 3x3x128.
model.add(
    MaxPooling2D(
        pool_size=(2, 2), 
        padding='same',
        name='maxPool3'))

#Flatten. Input = 3x3x128. Output = 1152.
model.add(Flatten(name='flatten'))

#Layer 4: Fully Connected. Input = 1152. Output = 120.        
model.add(Dense(120, activation='relu',name='layer4'))

#Dropout
if(p4<1.0):
    model.add(Dropout(p4,name='dropout4'))
    
#Layer 5: Fully Connected. Input = 120. Output = 84.
model.add(Dense(84, activation='relu',name='layer5'))
#Dropout
if(p5<1.0):
    model.add(Dropout(p5,name='dropout5'))
    
#Layer 5: Fully Connected. Input = 84. Output = n_classes.
model.add(Dense(n_classes, activation='softmax',name='layer5-out'))
        
model.summary()

#Optimizer choice
opt = tf.keras.optimizers.Adam(learning_rate=0.001)

#Compile the model
model.compile(optimizer=opt,
            loss=categorical_crossentropy,
            metrics=['accuracy'])

Model: "c132_c264_c3128_k3_p40.3_p50.2_esTrue"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
layer1 (Conv2D)              (None, 30, 30, 32)        320       
_________________________________________________________________
maxPool1 (MaxPooling2D)      (None, 15, 15, 32)        0         
_________________________________________________________________
layer2 (Conv2D)              (None, 13, 13, 64)        18496     
_________________________________________________________________
maxPool2 (MaxPooling2D)      (None, 7, 7, 64)          0         
_________________________________________________________________
layer3 (Conv2D)              (None, 5, 5, 128)         73856     
_________________________________________________________________
maxPool3 (MaxPooling2D)      (None, 3, 3, 128)         0         
_________________________________________________________________
flatten (Flatten)            (None, 1152)              0         
_________________________________________________________________
layer4 (Dense)               (None, 120)               138360    
_________________________________________________________________
dropout4 (Dropout)           (None, 120)               0         
_________________________________________________________________
layer5 (Dense)               (None, 84)                10164     
_________________________________________________________________
dropout5 (Dropout)           (None, 84)                0         
_________________________________________________________________
layer5-out (Dense)           (None, 43)                3655      
=================================================================
Total params: 244,851
Trainable params: 244,851
Non-trainable params: 0
_________________________________________________________________


if(train_model_flag):
    # Place the logs in a timestamped subdirectory
    # This allows to easy select different training runs
    # In order not to overwrite some data, it is useful to have a name with a timestamp
    time_stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
    complete_model_name = '{}-{}'.format(model.name, time_stamp)

    log_dir="logs/fit/" +complete_model_name

    # Specify the callback object
    # callback for tensorboard 
    tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)
    # early stop callback
    es_callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)


if(train_model_flag):
    if(early_stop_active):
        active_callbacks = [tensorboard_callback,es_callback]
    else:
        active_callbacks = [tensorboard_callback]
        
    history = model.fit(x=training_dataset_features, 
                        y=training_dataset_labels, 
                        epochs=training_epochs,
                        batch_size = training_batch_size,
                        validation_data=(X_valid, y_valid),
                        callbacks=active_callbacks,
                        verbose=1)

    tf.keras.models.save_model(model,'./models/{}'.format(complete_model_name))

    results = model.evaluate(X_test, 
                             y_test, 
                             batch_size=128,
                             verbose=0)
    print("test loss, test acc:", results)


    history.history

Epoch 1/200
338/338 [==============================] - 151s 445ms/step - loss: 2.6315 - accuracy: 0.2795 - val_loss: 0.3925 - val_accuracy: 0.8995
Epoch 2/200
338/338 [==============================] - 142s 419ms/step - loss: 0.9771 - accuracy: 0.7114 - val_loss: 0.1843 - val_accuracy: 0.9537
Epoch 3/200
338/338 [==============================] - 138s 408ms/step - loss: 0.6467 - accuracy: 0.8020 - val_loss: 0.1087 - val_accuracy: 0.9705
Epoch 4/200
338/338 [==============================] - 137s 404ms/step - loss: 0.4814 - accuracy: 0.8504 - val_loss: 0.0914 - val_accuracy: 0.9751
Epoch 5/200
338/338 [==============================] - 136s 401ms/step - loss: 0.3760 - accuracy: 0.8824 - val_loss: 0.0779 - val_accuracy: 0.9773
Epoch 6/200
338/338 [==============================] - 143s 424ms/step - loss: 0.3096 - accuracy: 0.9025 - val_loss: 0.0884 - val_accuracy: 0.9794
Epoch 7/200
338/338 [==============================] - 143s 422ms/step - loss: 0.2538 - accuracy: 0.9196 - val_loss: 0.0591 - val_accuracy: 0.9834
Epoch 8/200
338/338 [==============================] - 146s 433ms/step - loss: 0.2147 - accuracy: 0.9314 - val_loss: 0.0853 - val_accuracy: 0.9812
Epoch 9/200
338/338 [==============================] - 139s 412ms/step - loss: 0.1895 - accuracy: 0.9389 - val_loss: 0.0702 - val_accuracy: 0.9821
Epoch 10/200
338/338 [==============================] - 140s 413ms/step - loss: 0.1654 - accuracy: 0.9466 - val_loss: 0.0969 - val_accuracy: 0.9798
Epoch 11/200
338/338 [==============================] - 142s 421ms/step - loss: 0.1491 - accuracy: 0.9515 - val_loss: 0.0982 - val_accuracy: 0.9823
Epoch 12/200
338/338 [==============================] - 148s 437ms/step - loss: 0.1306 - accuracy: 0.9572 - val_loss: 0.0942 - val_accuracy: 0.9821
Epoch 13/200
338/338 [==============================] - 141s 417ms/step - loss: 0.1180 - accuracy: 0.9615 - val_loss: 0.1219 - val_accuracy: 0.9832
Epoch 14/200
338/338 [==============================] - 145s 427ms/step - loss: 0.1087 - accuracy: 0.9647 - val_loss: 0.0843 - val_accuracy: 0.9834
Epoch 15/200
338/338 [==============================] - 138s 409ms/step - loss: 0.0977 - accuracy: 0.9681 - val_loss: 0.1587 - val_accuracy: 0.9823
Epoch 16/200
338/338 [==============================] - 129s 381ms/step - loss: 0.0872 - accuracy: 0.9714 - val_loss: 0.1176 - val_accuracy: 0.9828
Epoch 17/200
338/338 [==============================] - 128s 378ms/step - loss: 0.0841 - accuracy: 0.9722 - val_loss: 0.1878 - val_accuracy: 0.9812
INFO:tensorflow:Assets written to: ./models/c132_c264_c3128_k3_p40.3_p50.2_esTrue-20210516-123943/assets
test loss, test acc: [0.21022966504096985, 0.9569279551506042]


%load_ext tensorboard 
%tensorboard --logdir logs/fit

Reusing TensorBoard on port 6007 (pid 35351), started 3 days, 0:12:26 ago. (Use '!kill 35351' to kill it.)


def resizeAndPad(pil_img,target_h,target_w):
    """
    Helper function to resize and pad sign pictures I took
    :param pil_img: pil image file
    :param target_h: target height
    :param target_w: target width
    :return: processed image 
    """  
    pil_img.thumbnail((target_h,target_w))

    width, height = pil_img.size

    if width == height:
        return pil_img
    elif width > height:
        result = Image.new(pil_img.mode, (width, width))
        result.paste(pil_img, (0, (width - height) // 2))
        return result
    else:
        result = Image.new(pil_img.mode, (height, height))
        result.paste(pil_img, ((height - width) // 2, 0))
        return result

    
def preprocess_own_pictures():
    """
    Helper function to process pictures I took and convert them
    in the right format. The function returns a dict similar to 
    the original traffic sign dataset
    :return: a dict for features and labels 
    """ 
    #make sure to be in the right directory
    os.chdir(notebook_root_path)
    os.chdir("./additional-signs")
    cwd = os.getcwd()
    #prepare the empty dict, keep the color version of the image
    #for visualization purposes
    data = {'features':np.empty([0,32,32,1]),
            'features_color':np.empty([0,32,32,3]),
            'labels':np.empty([0])}

    for file in glob.glob("*.png"):
        #process color image and add it to the dict
        color = Image.open(file).convert('RGB')
        im_new = resizeAndPad(color,32,32)
        pix = np.array(im_new)
        pix_exd = np.expand_dims(pix,axis=0)
        
        #add to the dict
        data['features_color'] = np.append(data['features_color'],pix_exd,axis=0)
        
        #process gray image nd add it to the dict
        gray = Image.open(file).convert('L')
        im_new = resizeAndPad(gray,32,32)
        pix = np.array(im_new)     
        pix_exd = np.expand_dims(np.expand_dims(pix,axis=2),axis=0)
        
        #add to the dict
        data['features'] = np.append(data['features'],pix_exd,axis=0)
        
        #add label to the dict
        sign_number = file.split('_')[1]
        data['labels'] = np.append(data['labels'],int(sign_number))
        
    os.chdir("../")
    return data


def writeup_single_image(color,gray,norm,label,img_num): 
    """
    Helper function to plot writeup images and save them
    :param color: color image
    :param gray: gray (CLAHE) image
    :param norm: normalised image
    :param label: image label
    :param img_num: image number to use as filename
    :return: save an image
    """ 
    plt.figure(figsize=(1,1))
    f,((ax1,ax2,ax3)) = plt.subplots(1,3,)
    f.tight_layout()
        
    ax1.imshow(color)
    ax1.set_title("Unprocessed image")

    ax2.imshow(gray,cmap='gray')
    ax2.set_title("CLAHE image")

    ax3.imshow(norm,cmap='gray')
    ax3.set_title("Normalised image")
    f.suptitle(label, fontsize=16)

    f.savefig('writeup_images/Own_image_{}.png'.format(img_num),
              dpi=300, 
              bbox_inches = "tight")


if (process_own_images):
    own_pics = preprocess_own_pictures()
    
    X_own_test_equal = applyEqualizationAndGray(
        np.uint8(own_pics['features_color']),
        gray_image=False)
    
    X_own_test_norm = normalise_input(X_own_test_equal)

    print("min:{},max:{}".format(np.min(X_own_test_norm),np.max(X_own_test_norm)))
    
    assert(len(X_own_test_norm) == len(own_pics['labels']))
    os.chdir(notebook_root_path)
    filename= '../traffic-signs-data/own_pictures_gray_norm.p'
    save_pickle_file(filename,X_own_test_norm,own_pics['labels'])

Mean: 134.888, Standard Deviation: 64.208
New Mean: -0.018, Standard Deviation: 0.762
min:-1.0,max:1.0


own_testing_file = '../traffic-signs-data/own_pictures_gray_norm.p'
with open(own_testing_file, mode='rb') as f:
    own_test = pickle.load(f) 

X_own_test,y_own_test = own_test['features'], own_test['labels']
y_own_test = to_categorical(y_own_test, n_classes)

assert(len(X_own_test) == len(y_own_test))
n_own_test = len(y_own_test)
print_stats(X_own_test,y_own_test,"own images")

Stats for the own images dataset
---------------------
Number of examples = 54
Image data shape = (32, 32, 1)
Number of classes = 43
---------------------


if(plot_writeup_images):
    
    plot_samples_from_all_classes(X_own_test,y_own_test,color='gray')
    
    plot_sign_count_bar(signnames,y_own_test,dataset_name='Own')
    
    #pick 5 unique pictures out the dataset
    selection_own_images = [0,1,2,7,9]
    for img_index in selection_own_images:
        color = (own_pics['features_color'][img_index]).astype(np.uint8)
        gray = X_own_test_equal[img_index]
        norm = X_own_test_norm[img_index]
        label = signnames[int(own_pics['labels'][img_index])]
        writeup_single_image(color,gray,norm,label,img_index)

Class 9: No passing                                          8 samples

--------------------------------------------------------------------------------------

Class 12: Priority road                                       9 samples

--------------------------------------------------------------------------------------

Class 13: Yield                                               3 samples

--------------------------------------------------------------------------------------

Class 17: No entry                                            6 samples

--------------------------------------------------------------------------------------

Class 26: Traffic signals                                     1 samples

--------------------------------------------------------------------------------------

Class 33: Turn right ahead                                    1 samples

--------------------------------------------------------------------------------------

Class 34: Turn left ahead                                     5 samples

--------------------------------------------------------------------------------------

Class 35: Ahead only                                          10 samples

--------------------------------------------------------------------------------------

Class 36: Go straight or right                                5 samples

--------------------------------------------------------------------------------------

Class 38: Keep right                                          3 samples


#model saved with early stop conditions
model_original_es = 'c132_c264_c3128_k3_p40.3_p50.2_esTrue-20210515-113756'
model_balanced_es = 'c132_c264_c3128_k3_p40.3_p50.2_esTrue-20210515-115926'
model_balanced_and_augmented_100_es = 'c132_c264_c3128_k3_p40.3_p50.2_esTrue-20210515-131619'

#choose the model to load and use for eval
model_file = model_balanced_and_augmented_100_es

loaded_model = tf.keras.models.load_model('./models/{}'.format(model_file))

loaded_model.summary()

Model: "c132_c264_c3128_k3_p40.3_p50.2_esTrue"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
layer1 (Conv2D)              (None, 30, 30, 32)        320       
_________________________________________________________________
maxPool1 (MaxPooling2D)      (None, 15, 15, 32)        0         
_________________________________________________________________
layer2 (Conv2D)              (None, 13, 13, 64)        18496     
_________________________________________________________________
maxPool2 (MaxPooling2D)      (None, 7, 7, 64)          0         
_________________________________________________________________
layer3 (Conv2D)              (None, 5, 5, 128)         73856     
_________________________________________________________________
maxPool3 (MaxPooling2D)      (None, 3, 3, 128)         0         
_________________________________________________________________
flatten (Flatten)            (None, 1152)              0         
_________________________________________________________________
layer4 (Dense)               (None, 120)               138360    
_________________________________________________________________
dropout4 (Dropout)           (None, 120)               0         
_________________________________________________________________
layer5 (Dense)               (None, 84)                10164     
_________________________________________________________________
dropout5 (Dropout)           (None, 84)                0         
_________________________________________________________________
layer5-out (Dense)           (None, 43)                3655      
=================================================================
Total params: 244,851
Trainable params: 244,851
Non-trainable params: 0
_________________________________________________________________


def pred_details(predictions,features,labels,signnames):
    """
    build an array where each element is a dictionary 
    that contains details about the prediction 
    :param predictions: predictions matrix size (dataset size x predictions) 
    :param features: dataset features
    :param labels: dataset labels
    :param signnames: string names of the traffic signs
    :return: array of dict
    """   
    
    results = []
    for i in range(len(predictions)):
        #get the position of the top 5 arguments
        top5 = predictions[i].argsort()[-5:][::-1]
        
        pred={'name':signnames[top5[0]],
              'number':top5[0],
              'certainty':predictions[i][top5[0]],
              'image':features[i].squeeze(),
              'correct_class':signnames[np.argmax(labels[i])],
              'predictions':predictions[i],
              'wrong_classification':np.argmax(labels[i]) != np.argmax(predictions[i]),
              'top5preds_idx' : top5,
              'top5preds_values' : predictions[i][top5],
              'top5preds_names': signnames[top5]
             }
        
        results.append(pred)
    return results

def plot_pred_and_image(height,bars,image,img_name,title,save_images=False):
    """
    Plot a 1x2 subplot with the top5 predictions and the
    corresponding image  
    :param height: values of top5 predictions 
    :param bars: corresponding traffic signs
    :param image: image to plot
    :param img_name: name of the image (the traffic sign)
    :param title: plot title
    :param save_images: flag for image saving
    :return: plots and save an image
    """   
    plt.figure(figsize=(3,3))
    f,((ax1,ax2)) = plt.subplots(1,2,)
    f.tight_layout(pad=5.0)

    # create dataset
    y_pos = np.arange(len(bars))

    # Create horizontal bars
    ax1.barh(y_pos, height)

    # Create names on the x-axis
    ax1.set_yticks(y_pos)
    ax1.set_yticklabels(bars)
    ax1.invert_yaxis()  # labels read top-to-bottom

    ax2.imshow(image,cmap='gray')
    ax2.set_title(img_name)

    f.suptitle(title, fontsize=16)

    # Show graphic
    plt.show()
    if(save_images):
        f.savefig('writeup_images/Wrong_pred_image_{}.png'.format(pred_index),
                  dpi=300, 
                  bbox_inches = "tight")

def plot_test_images(
    results,
    show_correct_preds=False,
    show_wrong_preds = False,
    save_images=False):

    """
    Organizes the plotting of predicted images and plots them
    :param results: all predictions 
    :param show_correct_preds: flag for showing correct predictions of signs
    :param show_wrong_preds: flag for showing wrong predictions of signs
    :param save_images: flag for image saving
    :return: plots and save an image
    """  

    for pred_index,pred in enumerate(results):

        if (show_wrong_preds):
            
            if pred['wrong_classification']:            
                print("Predicted class:", pred['name'])
                print("Instead of: ", pred['correct_class'])
                print("Certainty:", pred['certainty'])
                print("Correct prediction:", not pred['wrong_classification'])
                print("Top 5 predictions", pred['top5preds_names'])
                print("Top 5 predictions values", pred['top5preds_values'])
                #plot the results
                plot_pred_and_image(
                    pred['top5preds_values'],
                    pred['top5preds_names'],
                    pred['image'],
                    pred['correct_class'],
                    'Wrong prediction',
                    save_images )

                print("-------------------------------------\n")        
        
        elif(show_correct_preds):
            if pred['wrong_classification'] == False:            
                print("Predicted class:", pred['name'])
                print("Certainty:", pred['certainty'])
                print("Correct prediction:", not pred['wrong_classification'])
                print("Top 5 predictions", pred['top5preds_names'])
                print("Top 5 predictions values", pred['top5preds_values'])

                #plot the results
                plot_pred_and_image(
                    pred['top5preds_values'],
                    pred['top5preds_names'],
                    pred['image'],
                    pred['correct_class'],
                    'Correct prediction',
                    save_images )


                print("-------------------------------------\n")


### Run the predictions here and use the model to output the prediction for each image.
### Make sure to pre-process the images with the same pre-processing pipeline used earlier.
### Feel free to use as many code cells as needed.

# Evaluate the model on the test data using `evaluate`
results = loaded_model.evaluate(X_test, 
                         y_test, 
                         batch_size=128,
                         verbose=0)
print("test loss, test acc:", results)

results_own = loaded_model.evaluate(X_own_test, 
                         y_own_test,
                         verbose=0)
print("Own test loss, Own test acc:", results_own)

test loss, test acc: [0.17693284153938293, 0.9609659314155579]
Own test loss, Own test acc: [0.27811363339424133, 0.9444444179534912]


# Generate predictions (probabilities -- the output of the last layer)
# on new data using `predict`
print("Generate predictions for all samples")
predictions = loaded_model.predict(X_own_test,verbose=1)
details = pred_details(predictions,X_own_test,y_own_test,signnames)

Generate predictions for all samples
2/2 [==============================] - 0s 11ms/step


### Calculate the accuracy for the new images. 
### For example, if the model predicted 1 out of 5 signs correctly, 
### it's 20% accurate on these new images.
def eval_accuracy(results):
    """
    Evaluate the prediction accuracy of my own images 
    :param results: all predictions 
    :return: plots and save an image
    """  
    result = {'total':len(results),
              'correct':0,
              'wrong':0,
              'accuracy':0.0}
    correct = 0.0
    wrong = 0
    for pred in results:
        if(pred['wrong_classification']==False):
            correct += 1
        else:
            wrong+=1   
        
       
    result['correct'] =  correct
    result['wrong'] =  wrong
    result['accuracy'] = correct/len(results)
    return result

print(eval_accuracy(details))

{'total': 54, 'correct': 51.0, 'wrong': 3, 'accuracy': 0.9444444444444444}


### Print out the top five softmax probabilities for the predictions on the German traffic sign images found on the web. 
plot_test_images(details,
                 show_correct_preds=True,
                 show_wrong_preds=False,
                 save_images=False)

Predicted class: Priority road
Certainty: 1.0
Correct prediction: True
Top 5 predictions ['Priority road' 'No entry' 'End of speed limit (80km/h)'
 'Speed limit (100km/h)' 'Roundabout mandatory']
Top 5 predictions values [1.0000000e+00 1.9700419e-15 1.8342218e-16 2.2566294e-17 4.0773817e-18]

<Figure size 300x300 with 0 Axes>

-------------------------------------

Predicted class: Keep right
Certainty: 0.9922695
Correct prediction: True
Top 5 predictions ['Keep right' 'Turn left ahead' 'Go straight or left' 'No entry'
 'Go straight or right']
Top 5 predictions values [9.9226952e-01 7.1921796e-03 4.6006267e-04 3.8725608e-05 1.8978346e-05]

<Figure size 300x300 with 0 Axes>

-------------------------------------

Predicted class: Roundabout mandatory
Certainty: 0.9996617
Correct prediction: True
Top 5 predictions ['Roundabout mandatory' 'Turn left ahead' 'Keep right'
 'Speed limit (120km/h)' 'Vehicles over 3.5 metric tons prohibited']
Top 5 predictions values [9.9966168e-01 2.2072853e-04 4.8673264e-05 4.0549839e-05 2.1177562e-05]

<Figure size 300x300 with 0 Axes>

-------------------------------------

Predicted class: Roundabout mandatory
Certainty: 0.9999261
Correct prediction: True
Top 5 predictions ['Roundabout mandatory' 'Turn left ahead' 'Speed limit (120km/h)'
 'Vehicles over 3.5 metric tons prohibited' 'Keep right']
Top 5 predictions values [9.9992609e-01 4.6713114e-05 1.5399986e-05 4.8380912e-06 4.7025396e-06]

<Figure size 300x300 with 0 Axes>

-------------------------------------

Predicted class: Keep right
Certainty: 1.0
Correct prediction: True
Top 5 predictions ['Keep right' 'Turn right ahead' 'No entry' 'Turn left ahead' 'Keep left']
Top 5 predictions values [1.0000000e+00 1.4754780e-09 8.5058760e-10 2.5373992e-10 4.2124790e-11]

<Figure size 300x300 with 0 Axes>


#plot wrongly predicted images
plot_test_images(details,
                 show_correct_preds=False,
                 show_wrong_preds=True,
                 save_images=False)

Predicted class: Priority road
Instead of:  No entry
Certainty: 0.9837916
Correct prediction: False
Top 5 predictions ['Priority road' 'No entry'
 'End of no passing by vehicles over 3.5 metric tons' 'End of no passing'
 'Keep right']
Top 5 predictions values [9.8379159e-01 1.6207857e-02 1.8546235e-07 1.7226644e-07 6.1256536e-08]

<Figure size 300x300 with 0 Axes>

-------------------------------------

Predicted class: End of all speed and passing limits
Instead of:  No entry
Certainty: 0.54557186
Correct prediction: False
Top 5 predictions ['End of all speed and passing limits' 'No vehicles' 'Yield'
 'End of no passing' 'Priority road']
Top 5 predictions values [0.54557186 0.17437021 0.10472484 0.03869934 0.02921509]

<Figure size 300x300 with 0 Axes>

-------------------------------------

Predicted class: Ahead only
Instead of:  Go straight or right
Certainty: 0.9520398
Correct prediction: False
Top 5 predictions ['Ahead only' 'Go straight or right' 'Keep right' 'Turn right ahead'
 'Priority road']
Top 5 predictions values [9.5203978e-01 3.0644638e-02 1.6842106e-02 4.6105051e-04 7.7392333e-06]

<Figure size 300x300 with 0 Axes>

-------------------------------------


### Visualize your network's feature maps here.
### Feel free to use as many code cells as needed.

# image_input: the test image being fed into the network to produce the feature maps
# tf_activation: should be a tf variable name used during your training procedure that represents the calculated state of a specific weight layer
# activation_min/max: can be used to view the activation contrast in more detail, by default matplot sets min and max to the actual min and max values of the output
# plt_num: used to plot out multiple different weight feature map sets on the same block, just extend the plt number for each new feature map entry

def outputFeatureMap(image_input, tf_activation, activation_min=-1, activation_max=-1 ,plt_num=1):
    # Here make sure to preprocess your image_input in a way your network expects
    # with size, normalisation, ect if needed
    # image_input =
    # Note: x should be the same name as your network's tensorflow data placeholder variable
    # If you get an error tf_activation is not defined it may be having trouble accessing the variable from inside a function
    activation = tf_activation.eval(session=sess,feed_dict={x : image_input})
    featuremaps = activation.shape[3]
    plt.figure(plt_num, figsize=(15,15))
    for featuremap in range(featuremaps):
        plt.subplot(6,8, featuremap+1) # sets the number of feature maps to show on each row and column
        plt.title('FeatureMap ' + str(featuremap)) # displays the feature map number
        if activation_min != -1 & activation_max != -1:
            plt.imshow(activation[0,:,:, featuremap], interpolation="nearest", vmin =activation_min, vmax=activation_max, cmap="gray")
        elif activation_max != -1:
            plt.imshow(activation[0,:,:, featuremap], interpolation="nearest", vmax=activation_max, cmap="gray")
        elif activation_min !=-1:
            plt.imshow(activation[0,:,:, featuremap], interpolation="nearest", vmin=activation_min, cmap="gray")
        else:
            plt.imshow(activation[0,:,:, featuremap], interpolation="nearest", cmap="gray")

Project: Build a Traffic Sign Recognition Classifier¶

Part 1: load data, preprocessing, balancing and augmentations ¶

Pipeline Flags for writeup images generation, fake data generation and training¶

Load the dataset & check its integrity¶

Dataset Summary & Exploration¶

Visualization of some image examples¶

Distribution of datasets¶

Part 2: Deeplearning model design, data preprocessing, balancing, augmentations and model training ¶

Model Architecture¶

Input¶

Architecture¶

Output¶

Pre-process the Data Set¶

Normalisation and centering¶

Writeup image functions to show image augmentation filters¶

Apply equalization and gray filters¶

Apply histogram equalization to all datasets¶

dataset balancing and augmentation using imaug library¶

Generate fake data to balance all classes and increase the training dataset size¶

apply normalisation and centering to gray images¶

Loading pickle files with pre-processed datasets¶

Select the dataset to use for training and model parameters¶

define log folders and callback functions for model training¶

Train the model, check validation accuracy and test the results against the test datases¶

Use Tensorboard to¶

Part 3: Test a Model on New Images ¶

Load and Output the Images¶

load own images from pickle file¶

Predict the Sign Type for Each Image¶

Analyze Performance¶

Output Top 5 Softmax Probabilities For Each Additional Image I took¶

Project Writeup¶

Part 4 (Optional): Visualize the Neural Network's State with Test Images ¶

Project: Build a Traffic Sign Recognition Classifier¶

Part 1: load data, preprocessing, balancing and augmentations¶

Pipeline Flags for writeup images generation, fake data generation and training¶

Load the dataset & check its integrity¶

Dataset Summary & Exploration¶

Visualization of some image examples¶

Distribution of datasets¶

Part 2: Deeplearning model design, data preprocessing, balancing, augmentations and model training¶

Model Architecture¶

Input¶

Architecture¶

Output¶

Pre-process the Data Set¶

Normalisation and centering¶

Writeup image functions to show image augmentation filters¶

Apply equalization and gray filters¶

Apply histogram equalization to all datasets¶

dataset balancing and augmentation using imaug library¶

Generate fake data to balance all classes and increase the training dataset size¶

apply normalisation and centering to gray images¶

Loading pickle files with pre-processed datasets¶

Select the dataset to use for training and model parameters¶

define log folders and callback functions for model training¶

Train the model, check validation accuracy and test the results against the test datases¶

Use Tensorboard to¶

Part 3: Test a Model on New Images¶

Load and Output the Images¶

load own images from pickle file¶

Predict the Sign Type for Each Image¶

Analyze Performance¶

Output Top 5 Softmax Probabilities For Each Additional Image I took¶

Project Writeup¶

Part 4 (Optional): Visualize the Neural Network's State with Test Images¶

Part 1: load data, preprocessing, balancing and augmentations ¶

Part 2: Deeplearning model design, data preprocessing, balancing, augmentations and model training ¶

Part 3: Test a Model on New Images ¶

Part 4 (Optional): Visualize the Neural Network's State with Test Images ¶